def FE_pipeline(df):
## Creating duplicate features
df['image_top_1_C'] = df['image_top_1']
df['image_top_1_N'] = df['image_top_1']
print("Duplicated columns created.")
## Filling missing value
print("Filling missing value.")
# Categorical and Text Feature
df['param_1'] = df['param_1'].fillna('null')
df['param_2'] = df['param_2'].fillna('null')
df['param_3'] = df['param_3'].fillna('null')
df['description'] = df['description'].fillna('null')
df['image_top_1_C'] = df['image_top_1_C'].astype(str).fillna('null')
print("Missing value filled for categorical and text feature.")
# Numerical Feature
price_average_df = pd.read_csv('./bin/price_category_name_mean.csv')
image_top_1_average_df = pd.read_csv('./bin/image_top_1_N_category_name_mean.csv')
def fill_missing_price(df, price_average_df):
df = df.merge(price_average_df, on='category_name', how='left').reset_index(drop=True)
df.loc[df[df['price_x'].isnull() == True].index, 'price_x'] = df[df['price_x'].isnull() == True]['price_y']
df = df.drop(columns=['price_y'])
df.rename(columns = {'price_x':'price'}, inplace = True)
return df
def fill_missing_image_top_1(df, image_top_1_average_df):
df = df.merge(image_top_1_average_df, on='category_name', how='left').reset_index(drop=True)
df.loc[df[df['image_top_1_N_x'].isnull() == True].index, 'image_top_1_N_x'] = df[df['image_top_1_N_x'].isnull() == True]['image_top_1_N_y']
df = df.drop(columns=['image_top_1_N_y'])
df.rename(columns = {'image_top_1_N_x':'image_top_1_N'}, inplace = True)
return df
df = fill_missing_price(df, price_average_df)
df = fill_missing_image_top_1(df, image_top_1_average_df)
print("Missing value filled for numerical feature.")
# Image Feature
train_df_image_path = os.path.join('avito-demand-prediction','train_jpg')
def fill_missing_imagepath(x, main_path):
# missing images are filled with default_image.jpg
if x!=x:
return str(os.path.join(main_path, 'default_image.jpg'))
return os.path.join(main_path, 'avito-demand-prediction', str(x)+'.jpg')
df['image_fullpath'] = df['image'].apply(fill_missing_imagepath, main_path = train_df_image_path)
print("Missing value filled for Image feature.")
## Interactive Features
df['region__city'] = df['region'] + df['city']
df['param_1__param_2__param_3'] = df['param_1'] + df['param_2'] + df['param_3']
df['parent_category_name__user_type'] = df['parent_category_name'] + df['user_type']
df['category_name__user_type'] = df['category_name'] + df['user_type']
df['region__user_type'] = df['region'] + df['user_type']
print("Interactive features created.")
## More Features
# calling the text_featuring function which extract features from the title and description.
df["title_words_length"] = df["title"].apply(lambda x: len(x.split()))
df["description_words_length"] = df["description"].apply(lambda x: len(x.split()))
df['symbol1_count'] = df['description'].str.count('↓')
df['symbol2_count'] = df['description'].str.count('\*')
df['symbol3_count'] = df['description'].str.count('✔')
df['symbol4_count'] = df['description'].str.count('❀')
df['symbol5_count'] = df['description'].str.count('➚')
df['symbol6_count'] = df['description'].str.count('ஜ')
df['symbol7_count'] = df['description'].str.count('.')
df['symbol8_count'] = df['description'].str.count('!')
df['symbol9_count'] = df['description'].str.count('\?')
df['symbol10_count'] = df['description'].str.count(' ')
df['symbol11_count'] = df['description'].str.count('-')
df['symbol12_count'] = df['description'].str.count(',')
print("More features created.")
## Feature Engineering
# Categorical Features
def categorical_encoder_P(Series, name): # Pipeline
'''This function encode the categorical feature which we will use in NN along with embedding layer'''
tokeniser = pickle.load(open('./bin/'+name+'_tokeniser.pkl', 'rb'))
Series = np.array(tokeniser.texts_to_sequences(Series)).astype(np.int32)
Series = Series[:,0]
return Series
df['region'] = categorical_encoder_P(df['region'], 'region') #1
df['city'] = categorical_encoder_P(df['city'], 'city') #2
df['parent_category_name'] = categorical_encoder_P(df['parent_category_name'], 'parent_category_name') #3
df['category_name'] = categorical_encoder_P(df['category_name'], 'category_name') #4
df['param_1'] = categorical_encoder_P(df['param_1'], 'param_1') #5
df['param_2'] = categorical_encoder_P(df['param_2'], 'param_2') #6
df['param_3'] = categorical_encoder_P(df['param_3'], 'param_3') #7
df['user_type'] = categorical_encoder_P(df['user_type'], 'user_type') #8
df['region__city'] = categorical_encoder_P(df['region__city'], 'region__city') #9
df['param_1__param_2__param_3'] = categorical_encoder_P(df['param_1__param_2__param_3'], 'param_1__param_2__param_3') #10
df['parent_category_name__user_type'] = categorical_encoder_P(df['parent_category_name__user_type'], 'parent_category_name__user_type') #11
df['category_name__user_type'] = categorical_encoder_P(df['category_name__user_type'], 'category_name__user_type') #12
df['region__user_type'] = categorical_encoder_P(df['region__user_type'], 'region__user_type') #13
df['image_top_1_C'] = categorical_encoder_P(df['image_top_1_C'], 'image_top_1_C') #14
print("Encoding Categorical Features.")
# Numerical Feature
columns = ['price', 'item_seq_number', 'image_top_1_N', 'title_words_length',
'description_words_length',
'symbol1_count', 'symbol2_count', 'symbol3_count', 'symbol4_count',
'symbol5_count', 'symbol6_count', 'symbol7_count', 'symbol8_count',
'symbol9_count', 'symbol10_count', 'symbol11_count', 'symbol12_count']
# In this subsection we have will transform the numerical features.
# Applying logirthmic transformation to avoid bais caused by normalization. Then batch_normalization layer.
for col in columns:
df[col] = df[col].apply(lambda x: np.log10(x+1)) # adding 1 as bias
print("Numerical Features transformed")
# Text Feature Engineering
def text_clean(text):
'''This function clean the russian text'''
text = str(text)
text = text.lower()
clean = re.sub(r"[,.;@#?!&$-]+\ *", " ", text)
return clean
df['title'] = df['title'].apply(text_clean)
df['description'] = df['description'].apply(text_clean)
print("Text title and description are cleaned.")
with open("./bin/tokenizer_title.pkl", "rb") as input_file:
tokenizer_title = pickle.load(input_file)
with open("./bin/tokenizer_desc.pkl", "rb") as input_file:
tokenizer_desc = pickle.load(input_file)
def encoder(train, tokenizer, max_length):
''' This function perform the tokenization and then convert words to integers and then perform padding and returns the values '''
encoded_str = tokenizer.texts_to_sequences(train)
padded_str = np.array(pad_sequences(encoded_str, maxlen=max_length, padding='post')).astype(np.float64)
return padded_str
padded_title_df = encoder(df['title'], tokenizer_title, 8)
padded_desc_df = encoder(df['description'], tokenizer_desc, 200)
#### Image ##### ETA: 5-8 hrs
if False:
df = df.copy()
batch_size = 25000
maxlmt = df.shape[0]//batch_size + 1
print(maxlmt)
for i in range(0, maxlmt):
low = i*batch_size
high = min((i+1)*batch_size, df.shape[0])
print('Processing for:', low, high, end=', ')
if not os.path.exists(f'./image_feat/X_cv/export-{i}.csv'):
ddf = dd.from_pandas(df[['user_id', 'image_fullpath']][low:high], npartitions=30)
features_512 = ddf['image_fullpath'].apply(image_feat, model=model, meta=('image_fullpath', 'object'))
features_512 = pd.DataFrame(np.stack(features_512.compute().values), columns = [f'f{i}' for i in range(512)])
ddf = ddf.compute()
features_512.index = ddf.index
new_ddf = pd.concat([ddf,features_512], axis=1)
new_ddf.to_csv(f'./image_feat/X_cv/export-{i}.csv', index=False)
print('File Saved Success!')
else:
print('File Exists!')
else:
print("ALL Image Features for X_cv exists ./image_feat/X_cv/export-*.csv")
return df, padded_title_df, padded_desc_df
